{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0a2aee25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6147a413ede34f6694d5158a8151bc1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/107k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "122e42989ce7440cb400c1be57d1621c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b10331514144f3d8bd36aa411d355fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/624 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(PreTrainedTokenizer(name_or_path='bert-base-chinese', vocab_size=21128, model_max_len=512, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}),\n",
       " ['选择珠江花园的原因就是方便。',\n",
       "  '笔记本的键盘确实爽。',\n",
       "  '房间太小。其他的都一般。',\n",
       "  '今天才知道这书还有第6卷,真有点郁闷.',\n",
       "  '机器背面似乎被撕了张什么标签，残胶还在。'])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "#加载预训练字典和分词方法\n",
    "tokenizer = BertTokenizer.from_pretrained(\n",
    "    pretrained_model_name_or_path='bert-base-chinese',\n",
    "    cache_dir=None,\n",
    "    force_download=False,\n",
    ")\n",
    "\n",
    "sents = [\n",
    "    '选择珠江花园的原因就是方便。',\n",
    "    '笔记本的键盘确实爽。',\n",
    "    '房间太小。其他的都一般。',\n",
    "    '今天才知道这书还有第6卷,真有点郁闷.',\n",
    "    '机器背面似乎被撕了张什么标签，残胶还在。',\n",
    "]\n",
    "\n",
    "tokenizer, sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "286d64c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 6848, 2885, 4403, 3736, 5709, 1736, 4638, 1333, 1728, 2218, 3221, 3175, 912, 511, 102, 5011, 6381, 3315, 4638, 7241, 4669, 4802, 2141, 4272, 511, 102, 0, 0, 0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[CLS] 选 择 珠 江 花 园 的 原 因 就 是 方 便 。 [SEP] 笔 记 本 的 键 盘 确 实 爽 。 [SEP] [PAD] [PAD] [PAD]'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#编码两个句子\n",
    "out = tokenizer.encode(\n",
    "    text=sents[0],\n",
    "    text_pair=sents[1],\n",
    "\n",
    "    #当句子长度大于max_length时,截断\n",
    "    truncation=True,\n",
    "\n",
    "    #一律补pad到max_length长度\n",
    "    padding='max_length',\n",
    "    add_special_tokens=True,\n",
    "    max_length=30,\n",
    "    return_tensors=None,\n",
    ")\n",
    "\n",
    "print(out)\n",
    "\n",
    "tokenizer.decode(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e8f221a0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids : [101, 6848, 2885, 4403, 3736, 5709, 1736, 4638, 1333, 1728, 2218, 3221, 3175, 912, 511, 102, 5011, 6381, 3315, 4638, 7241, 4669, 4802, 2141, 4272, 511, 102, 0, 0, 0]\n",
      "token_type_ids : [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]\n",
      "special_tokens_mask : [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1]\n",
      "attention_mask : [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]\n",
      "length : 30\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[CLS] 选 择 珠 江 花 园 的 原 因 就 是 方 便 。 [SEP] 笔 记 本 的 键 盘 确 实 爽 。 [SEP] [PAD] [PAD] [PAD]'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#增强的编码函数\n",
    "out = tokenizer.encode_plus(\n",
    "    text=sents[0],\n",
    "    text_pair=sents[1],\n",
    "\n",
    "    #当句子长度大于max_length时,截断\n",
    "    truncation=True,\n",
    "\n",
    "    #一律补零到max_length长度\n",
    "    padding='max_length',\n",
    "    max_length=30,\n",
    "    add_special_tokens=True,\n",
    "\n",
    "    #可取值tf,pt,np,默认为返回list\n",
    "    return_tensors=None,\n",
    "\n",
    "    #返回token_type_ids\n",
    "    return_token_type_ids=True,\n",
    "\n",
    "    #返回attention_mask\n",
    "    return_attention_mask=True,\n",
    "\n",
    "    #返回special_tokens_mask 特殊符号标识\n",
    "    return_special_tokens_mask=True,\n",
    "\n",
    "    #返回offset_mapping 标识每个词的起止位置,这个参数只能BertTokenizerFast使用\n",
    "    #return_offsets_mapping=True,\n",
    "\n",
    "    #返回length 标识长度\n",
    "    return_length=True,\n",
    ")\n",
    "\n",
    "#input_ids 就是编码后的词\n",
    "#token_type_ids 第一个句子和特殊符号的位置是0,第二个句子的位置是1\n",
    "#special_tokens_mask 特殊符号的位置是1,其他位置是0\n",
    "#attention_mask pad的位置是0,其他位置是1\n",
    "#length 返回句子长度\n",
    "for k, v in out.items():\n",
    "    print(k, ':', v)\n",
    "\n",
    "tokenizer.decode(out['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1bb964a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids : [[101, 6848, 2885, 4403, 3736, 5709, 1736, 4638, 1333, 1728, 2218, 3221, 3175, 912, 102], [101, 5011, 6381, 3315, 4638, 7241, 4669, 4802, 2141, 4272, 511, 102, 0, 0, 0]]\n",
      "token_type_ids : [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
      "special_tokens_mask : [[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1]]\n",
      "length : [15, 12]\n",
      "attention_mask : [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('[CLS] 选 择 珠 江 花 园 的 原 因 就 是 方 便 [SEP]',\n",
       " '[CLS] 笔 记 本 的 键 盘 确 实 爽 。 [SEP] [PAD] [PAD] [PAD]')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#批量编码句子\n",
    "out = tokenizer.batch_encode_plus(\n",
    "    batch_text_or_text_pairs=[sents[0], sents[1]],\n",
    "    add_special_tokens=True,\n",
    "\n",
    "    #当句子长度大于max_length时,截断\n",
    "    truncation=True,\n",
    "\n",
    "    #一律补零到max_length长度\n",
    "    padding='max_length',\n",
    "    max_length=15,\n",
    "\n",
    "    #可取值tf,pt,np,默认为返回list\n",
    "    return_tensors=None,\n",
    "\n",
    "    #返回token_type_ids\n",
    "    return_token_type_ids=True,\n",
    "\n",
    "    #返回attention_mask\n",
    "    return_attention_mask=True,\n",
    "\n",
    "    #返回special_tokens_mask 特殊符号标识\n",
    "    return_special_tokens_mask=True,\n",
    "\n",
    "    #返回offset_mapping 标识每个词的起止位置,这个参数只能BertTokenizerFast使用\n",
    "    #return_offsets_mapping=True,\n",
    "\n",
    "    #返回length 标识长度\n",
    "    return_length=True,\n",
    ")\n",
    "\n",
    "#input_ids 就是编码后的词\n",
    "#token_type_ids 第一个句子和特殊符号的位置是0,第二个句子的位置是1\n",
    "#special_tokens_mask 特殊符号的位置是1,其他位置是0\n",
    "#attention_mask pad的位置是0,其他位置是1\n",
    "#length 返回句子长度\n",
    "for k, v in out.items():\n",
    "    print(k, ':', v)\n",
    "\n",
    "tokenizer.decode(out['input_ids'][0]), tokenizer.decode(out['input_ids'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "751e3052",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Be aware, overflowing tokens are not returned for the setting you have chosen, i.e. sequence pairs with the 'longest_first' truncation strategy. So the returned list will always be empty even if some tokens have been removed.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids : [[101, 6848, 2885, 4403, 3736, 5709, 1736, 4638, 1333, 1728, 2218, 3221, 3175, 912, 511, 102, 5011, 6381, 3315, 4638, 7241, 4669, 4802, 2141, 4272, 511, 102, 0, 0, 0], [101, 2791, 7313, 1922, 2207, 511, 1071, 800, 4638, 6963, 671, 5663, 511, 102, 791, 1921, 2798, 4761, 6887, 6821, 741, 6820, 3300, 5018, 127, 1318, 117, 4696, 3300, 102]]\n",
      "token_type_ids : [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]\n",
      "special_tokens_mask : [[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]]\n",
      "length : [27, 30]\n",
      "attention_mask : [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[CLS] 选 择 珠 江 花 园 的 原 因 就 是 方 便 。 [SEP] 笔 记 本 的 键 盘 确 实 爽 。 [SEP] [PAD] [PAD] [PAD]'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#批量编码成对的句子\n",
    "out = tokenizer.batch_encode_plus(\n",
    "    batch_text_or_text_pairs=[(sents[0], sents[1]), (sents[2], sents[3])],\n",
    "    add_special_tokens=True,\n",
    "\n",
    "    #当句子长度大于max_length时,截断\n",
    "    truncation=True,\n",
    "\n",
    "    #一律补零到max_length长度\n",
    "    padding='max_length',\n",
    "    max_length=30,\n",
    "\n",
    "    #可取值tf,pt,np,默认为返回list\n",
    "    return_tensors=None,\n",
    "\n",
    "    #返回token_type_ids\n",
    "    return_token_type_ids=True,\n",
    "\n",
    "    #返回attention_mask\n",
    "    return_attention_mask=True,\n",
    "\n",
    "    #返回special_tokens_mask 特殊符号标识\n",
    "    return_special_tokens_mask=True,\n",
    "\n",
    "    #返回offset_mapping 标识每个词的起止位置,这个参数只能BertTokenizerFast使用\n",
    "    #return_offsets_mapping=True,\n",
    "\n",
    "    #返回length 标识长度\n",
    "    return_length=True,\n",
    ")\n",
    "\n",
    "#input_ids 就是编码后的词\n",
    "#token_type_ids 第一个句子和特殊符号的位置是0,第二个句子的位置是1\n",
    "#special_tokens_mask 特殊符号的位置是1,其他位置是0\n",
    "#attention_mask pad的位置是0,其他位置是1\n",
    "#length 返回句子长度\n",
    "for k, v in out.items():\n",
    "    print(k, ':', v)\n",
    "\n",
    "tokenizer.decode(out['input_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2bbc2994",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(dict, 21128, False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#获取字典\n",
    "zidian = tokenizer.get_vocab()\n",
    "\n",
    "type(zidian), len(zidian), '月光' in zidian,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9ddd67b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(dict, 21131, 21128, 21130)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#添加新词\n",
    "tokenizer.add_tokens(new_tokens=['月光', '希望'])\n",
    "\n",
    "#添加新符号\n",
    "tokenizer.add_special_tokens({'eos_token': '[EOS]'})\n",
    "\n",
    "zidian = tokenizer.get_vocab()\n",
    "\n",
    "type(zidian), len(zidian), zidian['月光'], zidian['[EOS]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ed7cefad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 21128, 4638, 3173, 21129, 21130, 102, 0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'[CLS] 月光 的 新 希望 [EOS] [SEP] [PAD]'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#编码新添加的词\n",
    "out = tokenizer.encode(\n",
    "    text='月光的新希望[EOS]',\n",
    "    text_pair=None,\n",
    "\n",
    "    #当句子长度大于max_length时,截断\n",
    "    truncation=True,\n",
    "\n",
    "    #一律补pad到max_length长度\n",
    "    padding='max_length',\n",
    "    add_special_tokens=True,\n",
    "    max_length=8,\n",
    "    return_tensors=None,\n",
    ")\n",
    "\n",
    "print(out)\n",
    "\n",
    "tokenizer.decode(out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
